# encoding=utf-8
# -*- coding:utf-8 -*-
#Author: wxj0707@163.com

from Inc.Mysql import *
from Inc.Config import *
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from elasticsearch.helpers import bulk 
import jieba,sys,time,hashlib,Queue,threading
#jieba.load_userdict("data/dict.txt")                     #加载自定义词典  

reload(sys)
sys.setdefaultencoding('utf-8')

global q,es

#标点符号切割
def cutSymbol(str):
	data = []
	#print "完整原句: " + str
	str = str.replace('。','|').replace('!','!|').replace('?','?|') 
	#print "替换整句: " +str
	arr = str.split('|') #分隔完整句
	for v in arr:
		_v = v.encode('utf-8')
		data.append(_v)
		_v = _v.replace('，',',')
		_cv = _v.split(',') #分隔逗号
		if len(_cv) > 2:
			for j in _cv:
				data.append(j)
	return data


class MyThread(threading.Thread):
	def __init__(self,arg):
		super(MyThread, self).__init__()
		self.arg = arg
	def run(self):
		global q,es
		try:
			while True:
				v = q.get(timeout=3)
				print "now " + str(self.arg) + ", total:" + str(q.qsize())
				content = v['content']
				arr = cutSymbol(v['content'])
				ACTIONS = []
				for i in arr:
					seg_list = jieba.cut(i, cut_all=True)
					seg_lists = list(set(seg_list))
					for j in seg_lists:
						if len(j) > 1 :
							m2 = hashlib.md5()
							m2.update(j+i)
							id =  m2.hexdigest()
							#es.index(index="house", doc_type="house", id=id, body={"Word": j, "Sentence": i})
			  				#action = {"Word": j, "Sentence": i}
			  				action = {
					            "_index": "house",
					            "_type": "house",
					            "_source": {
					                  "Word":j,
					                  "Sentence":i}
					        }
							ACTIONS.append(action)
				success, _ = bulk(es, ACTIONS, index="house", raise_on_error=True)
				print('Performed %d actions' % success)
		except Exception, e:
			print str(self.arg) + " success"
			print e

if __name__ == '__main__':

	db = Mysql(user=Config().user,password=Config().password,host=Config().host)
	db.selectDb('letters')
	#data = db.queryAll("select * from thk_cn_new_product where tags like '%家居%'")
	data = db.queryAll("select * from thk_cn_new_product where tags like '%家居%'")
	#data = db.queryAll("select * from thk_cn_new_product where tags like '%家居%'")
	db.close()

	z = 0
	es = Elasticsearch([{'host':'192.168.2.28','port':9200}])

	q = Queue.LifoQueue()
	for i in data:
		z = z + 1
		if z > 7942:
			q.put(i)
	print q.qsize()

	for i in xrange(5):
		t = MyThread(i)
		t.start()
		
